﻿using System.Net;

namespace Fetches.Fetch.Fetcher.LinkExtractor
{
    /// <summary>
    /// TODO:解析Robots.txt并遵守
    /// </summary>
    public class RobotsText
    {
        /// <summary>
        /// 
        /// </summary>
        /// <param name="url">robots.txt文件的地址</param>
        public RobotsText(string url)
        {
            var content = new WebClient().DownloadString(url);
        }

        public bool IsPathAllowed(string url)
        {
            return true;
        }
    }
}
